import requests
# from my_base import MyBase
from fake_useragent import UserAgent
from lxml import etree
import re
import time

class BaiDuKuaiPai:
    ua = UserAgent().random
    headers = {
        'Host': 'www.baidu.com',
        'Connection': 'keep-alive',
        'Cache-Control': 'max-age=0',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': ua,
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Sec-Fetch-Site': 'same-origin',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-User': '?1',
        'Sec-Fetch-Dest': 'document',
        'Referer': 'https://www.baidu.com/',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9',
    }
    headers2 = {
        'User-Agent': ua,
    }
    def __init__(self,keyword):

        self.session = requests.session()
        self.keyword = keyword
        self.params =  {
            'wd':self.keyword,
            'usm':'2',
            'rsv_t':'98b3hbyLfnKq1zTa0nIcOn0syUN/JSisvzOUjiPO+Rtf79GMgTDG8UBaIpM', #变化的数据
            'rsv_pq':'b6a33410000ac1c4', #变化的数据
            'pn':'0',
            'oq':self.keyword,
            'ie':'utf-8'
        }

        self.params3 = {

        }

    def my_xpath(self,data,ruler): #xpath打包方法
        return etree.HTML(data).xpath(ruler)

    def request_next_page(self,url,host_url,name_url,a=0):
        shu = a
        li = []
        resp = self.session.get(url=url, headers=self.headers,params=self.params).text
        target_id_s = self.my_xpath(resp, '//*[@id="content_left"]/div[@class="result c-container "]/@id')
        res, name = '', ''
        for id in target_id_s:
            url = self.my_xpath(resp, '//div[@id="{}"]/div/div[2]/div[2]/a[1]/text()'.format(id))
            if not url:
                name = self.is_empty(self.my_xpath(resp,'//div[@id="{}"]/div[2]/span/text()'.format(id)))
            else:
                res = re.findall(r'{}'.format(host_url),url[0])
                if not res:
                    res = re.findall(r'{}'.format(host_url.replace('www.','')), url[0])
            if res or name == name_url:  # 这段代码可以封装
                target_url = self.is_empty(self.my_xpath(resp,'//*[@id="{}"]/h3/a/@href'.format(id)))
                print('---------------->'+target_url)
                result = self.session.get(target_url,headers=self.headers,params=self.params).text
                # print(result)
                ul = self.is_empty(re.findall(r'<noscript><META http-equiv="refresh" content="0;URL=\'(.*?)\'"></noscript>',result))
                self.headers['Referer'] = target_url
                print(ul)
                suju = self.session.get(url=ul,headers=self.headers2)
                # suju =''
                print(target_id_s)
                print(suju.status_code)
                return suju

        next_url = 'https://www.baidu.com' + self.is_empty(self.my_xpath(resp,'//*[@id="page"]/div/a[last()]/@href'))
        print(next_url)
        if shu<=10:
            shu += 1
            self.request_next_page(next_url,host_url,name_url,shu)

    def next_click(self):
        pass

    def get_interger(self, data): #转换数字
        try:
            res = int(data)
        except:
            try:
                res = float(data)
            except:
                res = data
        return res

    def is_empty(self, data):  # 处理数据为空的字段 str转int float
        if isinstance(data, list):
            if data != []:
                jieguo = data[0].strip()
                jieguo = self.get_interger(jieguo)
            else:
                jieguo = ''
        else:
            jieguo = self.get_interger(data)
        return jieguo

if __name__ == '__main__':
    bd = BaiDuKuaiPai('西红柿的功效与作用')
    bd.request_next_page('https://www.baidu.com/s?wd={}'.format(bd.keyword),'www.cndzys.com','九州醉')